{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import random\n",
    "import sentencepiece as spm\n",
    "from unidecode import unidecode\n",
    "from sklearn.utils import shuffle\n",
    "from tqdm import tqdm\n",
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "from malaya.text.function import transformer_textcleaning as cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v, sp_model):\n",
    "        self.vocab = v\n",
    "        self.sp_model = sp_model\n",
    "\n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(\n",
    "            self.sp_model, string, return_unicode = False, sample = False\n",
    "        )\n",
    "\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [self.sp_model.PieceToId(piece) for piece in tokens]\n",
    "\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [self.sp_model.IdToPiece(i) for i in ids]\n",
    "\n",
    "\n",
    "tokenizer = Tokenizer(v, sp_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(\n",
    "    'bert-base-2020-03-19/bert_config.json'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'bert-base-2020-03-19/model.ckpt-2000002'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 3\n",
    "batch_size = 16\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "_NEG_INF = -1e9\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        training = True,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.words,\n",
    "            input_mask=self.mask,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        self._arc_h = arc_h\n",
    "        self._arc_c = arc_c\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        self._type_h = type_h\n",
    "        self._type_c = type_c\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        self.out_arc = out_arc\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.decode_arc = decode_arc\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "#         self.decode_arc_t = tf.transpose(decode_arc, (0, 2, 1))\n",
    "#         sequence_loss_depends = tf.contrib.seq2seq.sequence_loss(logits = self.decode_arc_t,\n",
    "#                                                      targets = self.heads,\n",
    "#                                                      weights = mask)\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-66c1ff637216>:61: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From <ipython-input-10-66c1ff637216>:101: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-2020-03-19/model.ckpt-2000002\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.01369863, 0.06849315, 32.680145]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.02739726, 0.02739726, 152.1837]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 195576/195576 [12:15:03<00:00,  4.43it/s, accuracy=0.911, accuracy_depends=0.808, cost=0.425]   \n",
      "test minibatch loop: 100%|██████████| 625/625 [01:13<00:00,  8.45it/s, accuracy=0.883, accuracy_depends=0.8, cost=0.549]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.824410, training acc: 0.875569, training depends: 0.747707, valid loss: 0.470517, valid acc: 0.887186, valid depends: 0.798389\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 195576/195576 [12:20:26<00:00,  4.40it/s, accuracy=0.92, accuracy_depends=0.827, cost=0.29]     \n",
      "test minibatch loop: 100%|██████████| 625/625 [01:13<00:00,  8.51it/s, accuracy=0.895, accuracy_depends=0.833, cost=0.458]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.415683, training acc: 0.891427, training depends: 0.806351, valid loss: 0.393516, valid acc: 0.892850, valid depends: 0.813033\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 2\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'bert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 195576/195576 [11:52:11<00:00,  4.58it/s, accuracy=0.971, accuracy_depends=0.834, cost=2.2]    \n",
      "test minibatch loop: 100%|██████████| 625/625 [01:09<00:00,  9.00it/s, accuracy=0.945, accuracy_depends=0.828, cost=4.11] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 3.542283, training acc: 0.951459, training depends: 0.815969, valid loss: 3.709175, valid acc: 0.950416, valid depends: 0.814605\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 1\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'bert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    t = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "    return t, bert_tokens, [1] * len(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"193pt\" height=\"218pt\"\n",
       " viewBox=\"0.00 0.00 192.50 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 214)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-214 188.5,-214 188.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94,-173.9735C94,-162.1918 94,-146.5607 94,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"97.5001,-133.0033 94,-123.0034 90.5001,-133.0034 97.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"109\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M83.64,-86.9735C76.6677,-74.8418 67.3498,-58.6287 59.4999,-44.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"62.3642,-42.9295 54.3468,-36.0034 56.2951,-46.4175 62.3642,-42.9295\"/>\n",
       "<text text-anchor=\"middle\" x=\"92\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M105.3041,-86.7532C108.7732,-81.0866 112.5772,-74.8047 116,-69 120.611,-61.1804 125.5263,-52.6241 129.9752,-44.7918\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0613,-46.4448 134.9374,-36.0174 126.9682,-42.9989 133.0613,-46.4448\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f5adb1472e8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"2402pt\" height=\"914pt\"\n",
       " viewBox=\"0.00 0.00 2401.50 914.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 910)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-910 2397.5,-910 2397.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M402.5,-869.9735C402.5,-858.1918 402.5,-842.5607 402.5,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"406.0001,-829.0033 402.5,-819.0034 399.0001,-829.0034 406.0001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"417.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"339.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M389.4464,-782.9735C380.4924,-770.6085 368.4681,-754.0036 358.4613,-740.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"361.2369,-738.05 352.5369,-732.0034 355.5673,-742.1556 361.2369,-738.05\"/>\n",
       "<text text-anchor=\"middle\" x=\"387.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"506.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;13 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M424.0489,-782.9735C439.3878,-770.1419 460.1849,-752.7443 477.0541,-738.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"479.5543,-741.1043 484.9787,-732.0034 475.0629,-735.7352 479.5543,-741.1043\"/>\n",
       "<text text-anchor=\"middle\" x=\"495\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">parataxis</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"171.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M290.9409,-699.7702C274.1768,-694.0181 255.5788,-686.67 239.5,-678 225.8078,-670.6169 211.8619,-660.6667 200.1928,-651.5154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"202.1223,-648.5759 192.1345,-645.0411 197.738,-654.0329 202.1223,-648.5759\"/>\n",
       "<text text-anchor=\"middle\" x=\"260\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"257.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;4 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>13&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M454.907,-695.9735C411.9569,-680.9669 351.1329,-659.7151 308.3816,-644.7779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"309.3321,-641.4026 298.7373,-641.4082 307.0232,-648.0108 309.3321,-641.4026\"/>\n",
       "<text text-anchor=\"middle\" x=\"417.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"410.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;14 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>13&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M486.6087,-695.9735C472.5784,-683.2586 453.601,-666.0603 438.1064,-652.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"440.126,-649.1251 430.3658,-645.0034 435.4253,-654.3121 440.126,-649.1251\"/>\n",
       "<text text-anchor=\"middle\" x=\"475.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"565.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;33 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>13&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M518.7248,-695.9735C527.0312,-683.7252 538.1589,-667.3165 547.4768,-653.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"550.5748,-655.2442 553.2908,-645.0034 544.7814,-651.3153 550.5748,-655.2442\"/>\n",
       "<text text-anchor=\"middle\" x=\"563.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"686.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;37 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>13&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M550.408,-695.9997C563.4155,-690.448 577.621,-684.1675 590.5,-678 610.6478,-668.3515 632.6877,-656.7222 650.5626,-647.0066\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"652.4032,-649.9893 659.4983,-642.121 649.045,-643.8474 652.4032,-649.9893\"/>\n",
       "<text text-anchor=\"middle\" x=\"641\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"1038.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;39 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>13&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M568.0469,-703.935C670.2207,-687.2261 873.7373,-653.9443 976.3249,-637.1677\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"976.926,-640.616 986.2301,-635.5479 975.7962,-633.7078 976.926,-640.616\"/>\n",
       "<text text-anchor=\"middle\" x=\"818\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M216.2327,-610.13C202.0047,-604.2065 186.0027,-597.425 171.5,-591 150.9803,-581.9093 128.4889,-571.4679 109.2936,-562.4099\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"110.6944,-559.2007 100.1583,-558.0864 107.6999,-565.5279 110.6944,-559.2007\"/>\n",
       "<text text-anchor=\"middle\" x=\"183.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"170.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;7 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M232.8967,-608.8233C226.0462,-603.3668 218.7696,-597.1813 212.5,-591 204.7813,-583.39 197.038,-574.4488 190.3402,-566.1962\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"192.9892,-563.9039 184.0303,-558.2479 187.5067,-568.2562 192.9892,-563.9039\"/>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"257.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;8 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M257.5,-608.9735C257.5,-597.1918 257.5,-581.5607 257.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"261.0001,-568.0033 257.5,-558.0034 254.0001,-568.0034 261.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"279\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M62.5,-521.9735C62.5,-510.1918 62.5,-494.5607 62.5,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"66.0001,-481.0033 62.5,-471.0034 59.0001,-481.0034 66.0001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"74.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"169.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>8&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M239.2663,-521.9735C226.5232,-509.3752 209.3281,-492.3755 195.1976,-478.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"197.2824,-475.545 187.7103,-471.0034 192.361,-480.5229 197.2824,-475.545\"/>\n",
       "<text text-anchor=\"middle\" x=\"231.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"165.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M168.6712,-434.9735C168.1295,-423.1918 167.4108,-407.5607 166.7946,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"170.2834,-393.832 166.3277,-384.0034 163.2908,-394.1536 170.2834,-393.832\"/>\n",
       "<text text-anchor=\"middle\" x=\"180.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"165.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 10&#45;&gt;11 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>10&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M165.5,-347.9735C165.5,-336.1918 165.5,-320.5607 165.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"169.0001,-307.0033 165.5,-297.0034 162.0001,-307.0034 169.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"177.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"165.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M165.5,-260.9735C165.5,-249.1918 165.5,-233.5607 165.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"169.0001,-220.0033 165.5,-210.0034 162.0001,-220.0034 169.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"177.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"363.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;15 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>14&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M400.7616,-608.9735C394.2077,-596.8418 385.4488,-580.6287 378.0699,-566.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"381.0584,-565.138 373.226,-558.0034 374.8997,-568.4652 381.0584,-565.138\"/>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"473.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;20 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>14&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M423.5536,-608.9735C432.5076,-596.6085 444.5319,-580.0036 454.5387,-566.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"457.4327,-568.1556 460.4631,-558.0034 451.7631,-564.05 457.4327,-568.1556\"/>\n",
       "<text text-anchor=\"middle\" x=\"461.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"623.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M577.5176,-608.9735C585.6832,-596.7252 596.6223,-580.3165 605.7823,-566.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"608.8629,-568.2653 611.4978,-558.0034 603.0385,-564.3824 608.8629,-568.2653\"/>\n",
       "<text text-anchor=\"middle\" x=\"609.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"839.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;38 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>39&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M986.3077,-610.2093C970.1513,-604.5785 952.4192,-597.9507 936.5,-591 917.6686,-582.7778 897.4623,-572.4162 880.3927,-563.1836\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"881.7143,-559.9173 871.261,-558.1914 878.3564,-566.0594 881.7143,-559.9173\"/>\n",
       "<text text-anchor=\"middle\" x=\"956.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"949.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;41 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>39&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1015.0603,-608.9941C1008.2344,-603.4422 1000.8971,-597.1631 994.5,-591 986.3803,-583.1773 978.0574,-574.0951 970.8124,-565.7812\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"973.4037,-563.4263 964.2383,-558.1117 968.089,-567.982 973.4037,-563.4263\"/>\n",
       "<text text-anchor=\"middle\" x=\"1014.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"1038.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;40 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>39&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1038.5,-608.9735C1038.5,-597.1918 1038.5,-581.5607 1038.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1042.0001,-568.0033 1038.5,-558.0034 1035.0001,-568.0034 1042.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1059\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"1110.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;61 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>39&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1064.8871,-608.8513C1071.4537,-603.575 1078.1409,-597.4737 1083.5,-591 1089.3774,-583.9001 1094.5175,-575.3166 1098.6912,-567.2328\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1101.9105,-568.6155 1103.133,-558.0914 1095.6145,-565.5562 1101.9105,-568.6155\"/>\n",
       "<text text-anchor=\"middle\" x=\"1114\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"1211.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;46 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>39&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1090.7342,-611.124C1106.4559,-605.574 1123.5064,-598.7617 1138.5,-591 1153.0019,-583.4929 1167.9642,-573.4985 1180.5382,-564.3473\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1182.8882,-566.9617 1188.8267,-558.1876 1178.7127,-561.3433 1182.8882,-566.9617\"/>\n",
       "<text text-anchor=\"middle\" x=\"1183.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"1539.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;64 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>39&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1090.698,-617.9357C1183.7415,-601.7784 1377.8188,-568.0764 1477.4921,-550.7678\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1478.1658,-554.2033 1487.4195,-549.0439 1476.9681,-547.3065 1478.1658,-554.2033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1333\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"267.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;16 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>15&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M339.1403,-521.8946C332.0083,-516.34 324.3031,-510.0842 317.5,-504 308.5839,-496.026 299.2874,-486.8197 291.1633,-478.442\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"293.5205,-475.843 284.0766,-471.0404 288.4643,-480.684 293.5205,-475.843\"/>\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"363.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M450.7079,-521.9735C434.3365,-509.0253 412.0867,-491.4276 394.1591,-477.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"396.2774,-474.4616 386.2629,-471.0034 391.935,-479.952 396.2774,-474.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"433.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"473.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;21 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>20&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M473.5,-521.9735C473.5,-510.1918 473.5,-494.5607 473.5,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"477.0001,-481.0033 473.5,-471.0034 470.0001,-481.0034 477.0001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"485.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"591.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;24 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>20&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M497.9497,-521.9735C515.5908,-508.9669 539.5949,-491.269 558.8708,-477.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"561.1098,-479.7548 567.0816,-471.0034 556.9558,-474.1206 561.1098,-479.7548\"/>\n",
       "<text text-anchor=\"middle\" x=\"562\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"273.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;17 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>16&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M268.7432,-434.9735C269.5557,-423.1918 270.6337,-407.5607 271.5581,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"275.062,-394.2205 272.2584,-384.0034 268.0786,-393.7388 275.062,-394.2205\"/>\n",
       "<text text-anchor=\"middle\" x=\"283.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"273.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 17&#45;&gt;18 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>17&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M273.5,-347.9735C273.5,-336.1918 273.5,-320.5607 273.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"277.0001,-307.0033 273.5,-297.0034 270.0001,-307.0034 277.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"285.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 21&#45;&gt;22 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>21&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M458.7888,-434.9735C448.6026,-422.4919 434.8906,-405.6899 423.5506,-391.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"426.2267,-389.5379 417.1924,-384.0034 420.8035,-393.9638 426.2267,-389.5379\"/>\n",
       "<text text-anchor=\"middle\" x=\"454.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"493.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;23 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>24&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M560.3833,-434.8807C552.2639,-429.5618 543.7612,-423.4322 536.5,-417 528.091,-409.551 519.8809,-400.4249 512.9263,-391.962\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"515.6495,-389.7632 506.6776,-384.1278 510.177,-394.1281 515.6495,-389.7632\"/>\n",
       "<text text-anchor=\"middle\" x=\"557\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"572.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;25 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>24&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M587.5632,-434.9735C584.9902,-423.1918 581.5765,-407.5607 578.6495,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"581.9849,-393.0263 576.4318,-384.0034 575.1461,-394.5199 581.9849,-393.0263\"/>\n",
       "<text text-anchor=\"middle\" x=\"595.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"707.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;30 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>24&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M615.5353,-434.9735C632.8774,-421.9669 656.4746,-404.269 675.4238,-390.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"677.5955,-392.8034 683.4955,-384.0034 673.3955,-387.2034 677.5955,-392.8034\"/>\n",
       "<text text-anchor=\"middle\" x=\"667\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"572.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 25&#45;&gt;26 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>25&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M572.5,-347.9735C572.5,-336.1918 572.5,-320.5607 572.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.0001,-307.0033 572.5,-297.0034 569.0001,-307.0034 576.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"584.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"693.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.5992,-347.9735C702.7033,-336.1918 700.1879,-320.5607 698.0312,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"701.4415,-306.3203 696.3971,-297.0034 694.5304,-307.4324 701.4415,-306.3203\"/>\n",
       "<text text-anchor=\"middle\" x=\"721.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"809.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M728.6345,-347.9735C743.6784,-335.1419 764.0756,-317.7443 780.6204,-303.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"783.0556,-306.1558 788.3926,-297.0034 778.5129,-300.8299 783.0556,-306.1558\"/>\n",
       "<text text-anchor=\"middle\" x=\"776.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"572.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M572.5,-260.9735C572.5,-249.1918 572.5,-233.5607 572.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.0001,-220.0033 572.5,-210.0034 569.0001,-220.0034 576.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"584.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"572.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M572.5,-173.9735C572.5,-162.1918 572.5,-146.5607 572.5,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.0001,-133.0033 572.5,-123.0034 569.0001,-133.0034 576.0001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"584.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"809.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;32 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>31&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M809.5,-260.9735C809.5,-249.1918 809.5,-233.5607 809.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"813.0001,-220.0033 809.5,-210.0034 806.0001,-220.0034 813.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"846.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"773.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M654.5801,-521.9735C677.5081,-508.6753 708.8913,-490.473 733.6611,-476.1065\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"735.5654,-479.0482 742.4597,-471.0034 732.0534,-472.993 735.5654,-479.0482\"/>\n",
       "<text text-anchor=\"middle\" x=\"744.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"855.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 35&#45;&gt;36 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>35&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M790.4905,-434.9735C802.3647,-422.3752 818.3875,-405.3755 831.5546,-391.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"834.2194,-393.6811 838.5313,-384.0034 829.1254,-388.8799 834.2194,-393.6811\"/>\n",
       "<text text-anchor=\"middle\" x=\"839.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"958.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;43 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>41&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M951.3648,-521.9735C952.5836,-510.1918 954.2006,-494.5607 955.5871,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"959.0899,-481.3105 956.6376,-471.0034 952.1271,-480.5901 959.0899,-481.3105\"/>\n",
       "<text text-anchor=\"middle\" x=\"970.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1086.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 61&#45;&gt;63 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>61&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1105.5272,-521.9735C1102.2449,-510.0751 1097.8795,-494.2508 1094.1581,-480.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1097.4998,-479.7125 1091.4664,-471.0034 1090.7518,-481.5741 1097.4998,-479.7125\"/>\n",
       "<text text-anchor=\"middle\" x=\"1112.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"1202.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1209.6352,-521.9735C1208.4164,-510.1918 1206.7994,-494.5607 1205.4129,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1208.8729,-480.5901 1204.3624,-471.0034 1201.9101,-481.3105 1208.8729,-480.5901\"/>\n",
       "<text text-anchor=\"middle\" x=\"1237\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"1334.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1242.0097,-521.9319C1250.9965,-516.3783 1260.7598,-510.1138 1269.5,-504 1281.3013,-495.745 1293.8443,-486.12 1304.7319,-477.4697\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1307.0836,-480.0702 1312.6953,-471.0835 1302.7043,-474.6093 1307.0836,-480.0702\"/>\n",
       "<text text-anchor=\"middle\" x=\"1301.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1454.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>64&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1515.4239,-521.8061C1508.7253,-516.3493 1501.6151,-510.1679 1495.5,-504 1487.9414,-496.3761 1480.3763,-487.4313 1473.8395,-479.1796\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1476.5749,-476.9955 1467.6838,-471.2336 1471.0411,-481.2824 1476.5749,-476.9955\"/>\n",
       "<text text-anchor=\"middle\" x=\"1515.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"1539.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;65 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>64&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1539.5,-521.9735C1539.5,-510.1918 1539.5,-494.5607 1539.5,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1543.0001,-481.0033 1539.5,-471.0034 1536.0001,-481.0034 1543.0001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1560\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"1611.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;83 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>64&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1565.8871,-521.8513C1572.4537,-516.575 1579.1409,-510.4737 1584.5,-504 1590.3774,-496.9001 1595.5175,-488.3166 1599.6912,-480.2328\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1602.9105,-481.6155 1604.133,-471.0914 1596.6145,-478.5562 1602.9105,-481.6155\"/>\n",
       "<text text-anchor=\"middle\" x=\"1615\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"1720.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;68 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>64&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1591.5094,-523.6766C1607.2226,-518.1098 1624.3249,-511.4005 1639.5,-504 1655.5799,-496.1583 1672.4601,-485.8927 1686.624,-476.6287\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1688.6702,-479.4714 1695.0605,-471.0206 1684.795,-473.6418 1688.6702,-479.4714\"/>\n",
       "<text text-anchor=\"middle\" x=\"1684.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"958.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M958.5,-434.9735C958.5,-423.1918 958.5,-407.5607 958.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"962.0001,-394.0033 958.5,-384.0034 955.0001,-394.0034 962.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"966.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"1059.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;44 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>43&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M979.4273,-434.9735C994.3238,-422.1419 1014.5209,-404.7443 1030.9035,-390.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1033.3071,-393.1817 1038.5995,-384.0034 1028.7386,-387.878 1033.3071,-393.1817\"/>\n",
       "<text text-anchor=\"middle\" x=\"1027.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"1334.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1334.5,-434.9735C1334.5,-423.1918 1334.5,-407.5607 1334.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1338.0001,-394.0033 1334.5,-384.0034 1331.0001,-394.0034 1338.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1346.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"1157.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;49 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>48&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1292.201,-347.8705C1279.6591,-342.3152 1265.9513,-336.0651 1253.5,-330 1235.4622,-321.2137 1215.8499,-310.9295 1199.0815,-301.9069\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1200.5473,-298.7206 1190.0864,-297.0416 1197.2171,-304.8777 1200.5473,-298.7206\"/>\n",
       "<text text-anchor=\"middle\" x=\"1265.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"1256.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;50 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>48&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1311.3264,-347.9121C1304.9293,-342.457 1298.1887,-336.25 1292.5,-330 1285.7497,-322.5837 1279.1759,-313.8656 1273.533,-305.7558\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1276.3088,-303.6123 1267.8108,-297.2849 1270.5082,-307.5307 1276.3088,-303.6123\"/>\n",
       "<text text-anchor=\"middle\" x=\"1313\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"1339.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;51 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>48&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1335.536,-347.9735C1336.2131,-336.1918 1337.1115,-320.5607 1337.8817,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1341.3857,-307.1877 1338.4653,-297.0034 1334.3972,-306.786 1341.3857,-307.1877\"/>\n",
       "<text text-anchor=\"middle\" x=\"1359\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1449.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;57 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>48&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1360.0844,-347.8872C1367.9645,-342.2261 1376.6342,-335.9135 1384.5,-330 1395.8289,-321.4829 1408.0962,-311.9425 1418.8897,-303.4377\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1421.1431,-306.1179 1426.8163,-297.17 1416.8013,-300.627 1421.1431,-306.1179\"/>\n",
       "<text text-anchor=\"middle\" x=\"1423.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"1250.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 51&#45;&gt;52 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>51&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1319.4808,-260.8749C1313.3318,-255.2133 1306.5838,-248.9035 1300.5,-243 1292.0518,-234.8022 1282.9932,-225.6707 1274.9387,-217.4255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1277.3506,-214.8852 1267.8704,-210.1547 1272.3314,-219.7646 1277.3506,-214.8852\"/>\n",
       "<text text-anchor=\"middle\" x=\"1312.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"1346.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1427.0195,-260.8023C1420.0864,-255.1378 1412.4493,-248.8446 1405.5,-243 1395.4619,-234.5576 1384.5721,-225.2015 1374.9128,-216.8322\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1376.979,-213.9909 1367.134,-210.0758 1372.3887,-219.2758 1376.979,-213.9909\"/>\n",
       "<text text-anchor=\"middle\" x=\"1425.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1449.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;56 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>57&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1449.5,-260.9735C1449.5,-249.1918 1449.5,-233.5607 1449.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1453.0001,-220.0033 1449.5,-210.0034 1446.0001,-220.0034 1453.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1478\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"1584.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;59 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>57&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1481.1023,-260.9184C1490.6345,-255.3123 1501.0686,-249.0226 1510.5,-243 1523.8558,-234.4714 1538.2615,-224.6829 1550.7819,-215.9764\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1552.9235,-218.7497 1559.1126,-210.1504 1548.9118,-213.0132 1552.9235,-218.7497\"/>\n",
       "<text text-anchor=\"middle\" x=\"1556.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"1250.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;53 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>52&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1250.5,-173.9735C1250.5,-162.1918 1250.5,-146.5607 1250.5,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1254.0001,-133.0033 1250.5,-123.0034 1247.0001,-133.0034 1254.0001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1262.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"507.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 54&#45;&gt;54 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>54&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M555.1902,-895.6424C565.5134,-894.8951 573,-892.3477 573,-888 573,-885.2827 570.0756,-883.2686 565.3619,-881.9578\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"565.6126,-878.4542 555.1902,-880.3576 564.5248,-885.3692 565.6126,-878.4542\"/>\n",
       "<text text-anchor=\"middle\" x=\"585\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1528.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;58 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>59&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1572.8968,-173.9735C1565.0128,-161.7252 1554.4509,-145.3165 1545.6067,-131.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1548.4439,-129.5176 1540.0884,-123.0034 1542.5578,-133.3064 1548.4439,-129.5176\"/>\n",
       "<text text-anchor=\"middle\" x=\"1576.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"1639.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1595.896,-173.9735C1603.6393,-161.7252 1614.0126,-145.3165 1622.6987,-131.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1625.7333,-133.3262 1628.1186,-123.0034 1619.8165,-129.5857 1625.7333,-133.3262\"/>\n",
       "<text text-anchor=\"middle\" x=\"1626.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"1720.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;66 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>68&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1720.5,-434.9735C1720.5,-423.1918 1720.5,-407.5607 1720.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1724.0001,-394.0033 1720.5,-384.0034 1717.0001,-394.0034 1724.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1740.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"1822.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;69 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1743.0111,-434.8385C1749.9454,-429.1754 1757.5755,-422.874 1764.5,-417 1774.4056,-408.5972 1785.1199,-399.2507 1794.6124,-390.8781\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1797.0844,-393.3642 1802.2541,-384.1162 1792.4457,-388.1218 1797.0844,-393.3642\"/>\n",
       "<text text-anchor=\"middle\" x=\"1805\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"661.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 67&#45;&gt;67 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>67&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M707.5242,-895.6827C717.9063,-894.9906 725.5,-892.4297 725.5,-888 725.5,-885.2314 722.5337,-883.1929 717.7743,-881.8843\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"717.9383,-878.3688 707.5242,-880.3173 716.8804,-885.2884 717.9383,-878.3688\"/>\n",
       "<text text-anchor=\"middle\" x=\"754\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"1821.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;70 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>69&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1822.2928,-347.9735C1822.1574,-336.1918 1821.9777,-320.5607 1821.8237,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1825.3217,-306.9624 1821.7069,-297.0034 1818.3222,-307.043 1825.3217,-306.9624\"/>\n",
       "<text text-anchor=\"middle\" x=\"1833.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"1950.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;75 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>69&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1849.0217,-347.9735C1868.3295,-334.8503 1894.6638,-316.9512 1915.6669,-302.6756\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1917.7093,-305.5194 1924.0123,-297.0034 1913.7743,-299.7301 1917.7093,-305.5194\"/>\n",
       "<text text-anchor=\"middle\" x=\"1905.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"1721.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;71 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>70&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1771.6245,-260.8778C1762.5388,-256.0371 1753.6784,-250.1194 1746.5,-243 1739.9581,-236.5119 1734.8613,-227.9659 1731.0261,-219.7285\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1734.1856,-218.2122 1727.0924,-210.3421 1727.7296,-220.9178 1734.1856,-218.2122\"/>\n",
       "<text text-anchor=\"middle\" x=\"1783.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"1827.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;73 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>70&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1822.7432,-260.9735C1823.5557,-249.1918 1824.6337,-233.5607 1825.5581,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1829.062,-220.2205 1826.2584,-210.0034 1822.0786,-219.7388 1829.062,-220.2205\"/>\n",
       "<text text-anchor=\"middle\" x=\"1836.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"1950.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;74 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>75&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1950.5,-260.9735C1950.5,-249.1918 1950.5,-233.5607 1950.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1954.0001,-220.0033 1950.5,-210.0034 1947.0001,-220.0034 1954.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1966.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"2092.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;77 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1979.9225,-260.9735C2001.5325,-247.7336 2031.0767,-229.6326 2054.477,-215.2958\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2056.4168,-218.212 2063.1152,-210.0034 2052.7598,-212.2432 2056.4168,-218.212\"/>\n",
       "<text text-anchor=\"middle\" x=\"2041\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"1827.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;72 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>73&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1827.5,-173.9735C1827.5,-162.1918 1827.5,-146.5607 1827.5,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1831.0001,-133.0033 1827.5,-123.0034 1824.0001,-133.0034 1831.0001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1843.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"1982.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>77&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2069.7079,-173.9735C2053.3365,-161.0253 2031.0867,-143.4276 2013.1591,-129.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2015.2774,-126.4616 2005.2629,-123.0034 2010.935,-131.952 2015.2774,-126.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"2064.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"2092.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>77&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2092.5,-173.9735C2092.5,-162.1918 2092.5,-146.5607 2092.5,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2096.0001,-133.0033 2092.5,-123.0034 2089.0001,-133.0034 2096.0001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2103.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"2215.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;80 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>77&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2117.9857,-173.9735C2136.4568,-160.9086 2161.6201,-143.1102 2181.7582,-128.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2183.9039,-131.6355 2190.047,-123.0034 2179.8617,-125.9206 2183.9039,-131.6355\"/>\n",
       "<text text-anchor=\"middle\" x=\"2198.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"2084.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;79 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>78&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2090.8424,-86.9735C2089.759,-75.1918 2088.3217,-59.5607 2087.0892,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2090.5565,-45.6408 2086.1555,-36.0034 2083.586,-46.2819 2090.5565,-45.6408\"/>\n",
       "<text text-anchor=\"middle\" x=\"2109.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"2215.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 80&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>80&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2215.5,-86.9735C2215.5,-75.1918 2215.5,-59.5607 2215.5,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2219.0001,-46.0033 2215.5,-36.0034 2212.0001,-46.0034 2219.0001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2252.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"2346.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 80&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>80&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2261.4389,-86.8926C2272.3359,-81.7694 2283.6477,-75.73 2293.5,-69 2304.2091,-61.6847 2314.7628,-52.1467 2323.5989,-43.301\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2326.135,-45.7134 2330.583,-36.0975 2321.1092,-40.8408 2326.135,-45.7134\"/>\n",
       "<text text-anchor=\"middle\" x=\"2330.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f59ed53e518>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bert-base-dependency/model.ckpt'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'bert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word, training = False)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'bert-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 625/625 [01:08<00:00,  9.15it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    0.99944   1.00000   0.99972    481710\n",
      "            X    1.00000   0.99690   0.99845     61945\n",
      "          acl    0.84049   0.83081   0.83562      3298\n",
      "        advcl    0.66176   0.62500   0.64286      1656\n",
      "       advmod    0.95277   0.94836   0.95056      6700\n",
      "         amod    0.91014   0.90522   0.90767      4621\n",
      "        appos    0.81969   0.77457   0.79650      3052\n",
      "         case    0.98062   0.98199   0.98130     11492\n",
      "           cc    0.98549   0.97787   0.98166      3750\n",
      "        ccomp    0.56966   0.48042   0.52125       383\n",
      "     compound    0.90827   0.93299   0.92047     11133\n",
      "compound:plur    0.55882   0.63333   0.59375        30\n",
      "         conj    0.89626   0.88883   0.89253      5424\n",
      "          cop    0.96823   0.96661   0.96742       599\n",
      "        csubj    0.66667   0.40000   0.50000        10\n",
      "   csubj:pass    0.00000   0.00000   0.00000         1\n",
      "          dep    0.65768   0.67218   0.66485       363\n",
      "          det    0.94642   0.92567   0.93593      3969\n",
      "        fixed    0.93103   0.78947   0.85443       171\n",
      "         flat    0.96042   0.96700   0.96370     18393\n",
      "         iobj    0.00000   0.00000   0.00000         4\n",
      "         mark    0.91283   0.93645   0.92449      1778\n",
      "         nmod    0.86204   0.85472   0.85836      4591\n",
      "        nsubj    0.86476   0.86536   0.86506      7145\n",
      "   nsubj:pass    0.81694   0.79646   0.80657      2034\n",
      "       nummod    0.96463   0.95920   0.96191      4436\n",
      "          obj    0.90765   0.91968   0.91363      6412\n",
      "          obl    0.88052   0.86034   0.87031      5191\n",
      "    parataxis    0.43636   0.38710   0.41026       372\n",
      "        punct    0.99723   0.99574   0.99649     20643\n",
      "         root    0.89938   0.91530   0.90727     10000\n",
      "        xcomp    0.74105   0.77125   0.75585      1718\n",
      "\n",
      "     accuracy                        0.98402    683024\n",
      "    macro avg    0.79679   0.77996   0.78684    683024\n",
      " weighted avg    0.98390   0.98402   0.98394    683024\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.8204592062639473\n",
      "types accuracy: 0.7997014021904779\n",
      "root accuracy: 0.9893670748261997\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-32-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 214 variables.\n",
      "INFO:tensorflow:Converted 214 variables to const ops.\n",
      "2768 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('bert-base-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-35-96a697e0d332>:6: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'bert-base-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
